{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Please input your directory for the top level folder\n",
    "folder name : SUBMISSION MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_ = \"INPUT-PROJECT-DIRECTORY/submission_model/\"  # input only here"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### setting other directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data_dir = dir_ + \"2. data/\"\n",
    "processed_data_dir = dir_ + \"2. data/processed/\"\n",
    "log_dir = dir_ + \"4. logs/\"\n",
    "model_dir = dir_ + \"5. models/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################################################################################\n",
    "####################### 1-2. recursive model by store & cat ########################\n",
    "####################################################################################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ver, KKK = \"priv\", 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "STORES = [\n",
    "    \"CA_1\",\n",
    "    \"CA_2\",\n",
    "    \"CA_3\",\n",
    "    \"CA_4\",\n",
    "    \"TX_1\",\n",
    "    \"TX_2\",\n",
    "    \"TX_3\",\n",
    "    \"WI_1\",\n",
    "    \"WI_2\",\n",
    "    \"WI_3\",\n",
    "]\n",
    "CATS = [\"HOBBIES\", \"HOUSEHOLD\", \"FOODS\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# General imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os, sys, gc, time, warnings, pickle, psutil, random\n",
    "\n",
    "# custom imports\n",
    "from multiprocessing import Pool\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helpers\n",
    "#################################################################################\n",
    "def seed_everything(seed=0):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "\n",
    "## Multiprocess Runs\n",
    "def df_parallelize_run(func, t_split):\n",
    "    num_cores = np.min([N_CORES, len(t_split)])\n",
    "    pool = Pool(num_cores)\n",
    "    df = pd.concat(pool.map(func, t_split), axis=1)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Helper to load data by store ID\n",
    "#################################################################################\n",
    "# Read data\n",
    "def get_data_by_store(store, dept):\n",
    "\n",
    "    df = pd.concat(\n",
    "        [\n",
    "            pd.read_pickle(BASE),\n",
    "            pd.read_pickle(PRICE).iloc[:, 2:],\n",
    "            pd.read_pickle(CALENDAR).iloc[:, 2:],\n",
    "        ],\n",
    "        axis=1,\n",
    "    )\n",
    "\n",
    "    df = df[df[\"d\"] >= START_TRAIN]\n",
    "\n",
    "    df = df[(df[\"store_id\"] == store) & (df[\"cat_id\"] == dept)]\n",
    "\n",
    "    df2 = pd.read_pickle(MEAN_ENC)[mean_features]\n",
    "    df2 = df2[df2.index.isin(df.index)]\n",
    "\n",
    "    df3 = pd.read_pickle(LAGS).iloc[:, 3:]\n",
    "    df3 = df3[df3.index.isin(df.index)]\n",
    "\n",
    "    df = pd.concat([df, df2], axis=1)\n",
    "    del df2\n",
    "\n",
    "    df = pd.concat([df, df3], axis=1)\n",
    "    del df3\n",
    "\n",
    "    features = [col for col in list(df) if col not in remove_features]\n",
    "    df = df[[\"id\", \"d\", TARGET] + features]\n",
    "\n",
    "    df = df.reset_index(drop=True)\n",
    "\n",
    "    return df, features\n",
    "\n",
    "\n",
    "# Recombine Test set after training\n",
    "def get_base_test():\n",
    "    base_test = pd.DataFrame()\n",
    "\n",
    "    for store_id in STORES:\n",
    "        for state_id in CATS:\n",
    "            temp_df = pd.read_pickle(\n",
    "                processed_data_dir\n",
    "                + \"test_\"\n",
    "                + store_id\n",
    "                + \"_\"\n",
    "                + state_id\n",
    "                + \".pkl\"\n",
    "            )\n",
    "            temp_df[\"store_id\"] = store_id\n",
    "            temp_df[\"cat_id\"] = state_id\n",
    "            base_test = pd.concat([base_test, temp_df]).reset_index(drop=True)\n",
    "\n",
    "    return base_test\n",
    "\n",
    "\n",
    "########################### Helper to make dynamic rolling lags\n",
    "#################################################################################\n",
    "def make_lag(LAG_DAY):\n",
    "    lag_df = base_test[[\"id\", \"d\", TARGET]]\n",
    "    col_name = \"sales_lag_\" + str(LAG_DAY)\n",
    "    lag_df[col_name] = (\n",
    "        lag_df.groupby([\"id\"])[TARGET]\n",
    "        .transform(lambda x: x.shift(LAG_DAY))\n",
    "        .astype(np.float16)\n",
    "    )\n",
    "    return lag_df[[col_name]]\n",
    "\n",
    "\n",
    "def make_lag_roll(LAG_DAY):\n",
    "    shift_day = LAG_DAY[0]\n",
    "    roll_wind = LAG_DAY[1]\n",
    "    lag_df = base_test[[\"id\", \"d\", TARGET]]\n",
    "    col_name = \"rolling_mean_tmp_\" + str(shift_day) + \"_\" + str(roll_wind)\n",
    "    lag_df[col_name] = lag_df.groupby([\"id\"])[TARGET].transform(\n",
    "        lambda x: x.shift(shift_day).rolling(roll_wind).mean()\n",
    "    )\n",
    "    return lag_df[[col_name]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Model params\n",
    "#################################################################################\n",
    "import lightgbm as lgb\n",
    "\n",
    "lgb_params = {\n",
    "    \"boosting_type\": \"gbdt\",\n",
    "    \"objective\": \"tweedie\",\n",
    "    \"tweedie_variance_power\": 1.1,\n",
    "    \"metric\": \"rmse\",\n",
    "    \"subsample\": 0.5,\n",
    "    \"subsample_freq\": 1,\n",
    "    \"learning_rate\": 0.015,\n",
    "    \"num_leaves\": 2**8 - 1,\n",
    "    \"min_data_in_leaf\": 2**8 - 1,\n",
    "    \"feature_fraction\": 0.5,\n",
    "    \"max_bin\": 100,\n",
    "    \"n_estimators\": 3000,\n",
    "    \"boost_from_average\": False,\n",
    "    \"verbose\": -1,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "########################### Vars\n",
    "#################################################################################\n",
    "VER = 1\n",
    "SEED = 42\n",
    "seed_everything(SEED)\n",
    "lgb_params[\"seed\"] = SEED\n",
    "N_CORES = psutil.cpu_count()\n",
    "\n",
    "\n",
    "# LIMITS and const\n",
    "TARGET = \"sales\"\n",
    "START_TRAIN = 700\n",
    "END_TRAIN = 1941 - 28 * KKK\n",
    "P_HORIZON = 28\n",
    "USE_AUX = False\n",
    "\n",
    "remove_features = [\n",
    "    \"id\",\n",
    "    \"cat_id\",\n",
    "    \"state_id\",\n",
    "    \"store_id\",\n",
    "    \"date\",\n",
    "    \"wm_yr_wk\",\n",
    "    \"d\",\n",
    "    TARGET,\n",
    "]\n",
    "mean_features = [\n",
    "    \"enc_store_id_dept_id_mean\",\n",
    "    \"enc_store_id_dept_id_std\",\n",
    "    \"enc_item_id_store_id_mean\",\n",
    "    \"enc_item_id_store_id_std\",\n",
    "]\n",
    "\n",
    "ORIGINAL = raw_data_dir\n",
    "BASE = processed_data_dir + \"grid_part_1.pkl\"\n",
    "PRICE = processed_data_dir + \"grid_part_2.pkl\"\n",
    "CALENDAR = processed_data_dir + \"grid_part_3.pkl\"\n",
    "LAGS = processed_data_dir + \"lags_df_28.pkl\"\n",
    "MEAN_ENC = processed_data_dir + \"mean_encoding_df.pkl\"\n",
    "\n",
    "\n",
    "SHIFT_DAY = 28\n",
    "N_LAGS = 15\n",
    "LAGS_SPLIT = [col for col in range(SHIFT_DAY, SHIFT_DAY + N_LAGS)]\n",
    "ROLS_SPLIT = []\n",
    "for i in [1, 7, 14]:\n",
    "    for j in [7, 14, 30, 60]:\n",
    "        ROLS_SPLIT.append([i, j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train CA_1 HOBBIES\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.16414\n",
      "[200]\tvalid_0's rmse: 2.08886\n",
      "[300]\tvalid_0's rmse: 2.04793\n",
      "[400]\tvalid_0's rmse: 2.0046\n",
      "[500]\tvalid_0's rmse: 1.96335\n",
      "[600]\tvalid_0's rmse: 1.92071\n",
      "[700]\tvalid_0's rmse: 1.88306\n",
      "[800]\tvalid_0's rmse: 1.84507\n",
      "[900]\tvalid_0's rmse: 1.80932\n",
      "[1000]\tvalid_0's rmse: 1.77228\n",
      "[1100]\tvalid_0's rmse: 1.73566\n",
      "[1200]\tvalid_0's rmse: 1.69836\n",
      "[1300]\tvalid_0's rmse: 1.66419\n",
      "[1400]\tvalid_0's rmse: 1.62874\n",
      "[1500]\tvalid_0's rmse: 1.5964\n",
      "[1600]\tvalid_0's rmse: 1.56647\n",
      "[1700]\tvalid_0's rmse: 1.53748\n",
      "[1800]\tvalid_0's rmse: 1.50902\n",
      "[1900]\tvalid_0's rmse: 1.48282\n",
      "[2000]\tvalid_0's rmse: 1.45599\n",
      "[2100]\tvalid_0's rmse: 1.42992\n",
      "[2200]\tvalid_0's rmse: 1.40797\n",
      "[2300]\tvalid_0's rmse: 1.38649\n",
      "[2400]\tvalid_0's rmse: 1.36769\n",
      "[2500]\tvalid_0's rmse: 1.34734\n",
      "[2600]\tvalid_0's rmse: 1.32939\n",
      "[2700]\tvalid_0's rmse: 1.31148\n",
      "[2800]\tvalid_0's rmse: 1.29578\n",
      "[2900]\tvalid_0's rmse: 1.28107\n",
      "[3000]\tvalid_0's rmse: 1.26571\n",
      "Train CA_1 HOUSEHOLD\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 1.25212\n",
      "[200]\tvalid_0's rmse: 1.19975\n",
      "[300]\tvalid_0's rmse: 1.1893\n",
      "[400]\tvalid_0's rmse: 1.18055\n",
      "[500]\tvalid_0's rmse: 1.1734\n",
      "[600]\tvalid_0's rmse: 1.16612\n",
      "[700]\tvalid_0's rmse: 1.16012\n",
      "[800]\tvalid_0's rmse: 1.15384\n",
      "[900]\tvalid_0's rmse: 1.14841\n",
      "[1000]\tvalid_0's rmse: 1.14296\n",
      "[1100]\tvalid_0's rmse: 1.13743\n",
      "[1200]\tvalid_0's rmse: 1.13215\n",
      "[1300]\tvalid_0's rmse: 1.12748\n",
      "[1400]\tvalid_0's rmse: 1.12309\n",
      "[1500]\tvalid_0's rmse: 1.1188\n",
      "[1600]\tvalid_0's rmse: 1.11438\n",
      "[1700]\tvalid_0's rmse: 1.11013\n",
      "[1800]\tvalid_0's rmse: 1.10538\n",
      "[1900]\tvalid_0's rmse: 1.1014\n",
      "[2000]\tvalid_0's rmse: 1.09758\n",
      "[2100]\tvalid_0's rmse: 1.09447\n",
      "[2200]\tvalid_0's rmse: 1.09046\n",
      "[2300]\tvalid_0's rmse: 1.08659\n",
      "[2400]\tvalid_0's rmse: 1.08347\n",
      "[2500]\tvalid_0's rmse: 1.08002\n",
      "[2600]\tvalid_0's rmse: 1.07676\n",
      "[2700]\tvalid_0's rmse: 1.07313\n",
      "[2800]\tvalid_0's rmse: 1.06947\n",
      "[2900]\tvalid_0's rmse: 1.06548\n",
      "[3000]\tvalid_0's rmse: 1.06254\n",
      "Train CA_1 FOODS\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.65329\n",
      "[200]\tvalid_0's rmse: 2.33343\n",
      "[300]\tvalid_0's rmse: 2.29563\n",
      "[400]\tvalid_0's rmse: 2.27611\n",
      "[500]\tvalid_0's rmse: 2.26152\n",
      "[600]\tvalid_0's rmse: 2.24957\n",
      "[700]\tvalid_0's rmse: 2.23952\n",
      "[800]\tvalid_0's rmse: 2.23127\n",
      "[900]\tvalid_0's rmse: 2.22144\n",
      "[1000]\tvalid_0's rmse: 2.21289\n",
      "[1100]\tvalid_0's rmse: 2.20414\n",
      "[1200]\tvalid_0's rmse: 2.19767\n",
      "[1300]\tvalid_0's rmse: 2.1892\n",
      "[1400]\tvalid_0's rmse: 2.18311\n",
      "[1500]\tvalid_0's rmse: 2.17698\n",
      "[1600]\tvalid_0's rmse: 2.17134\n",
      "[1700]\tvalid_0's rmse: 2.16566\n",
      "[1800]\tvalid_0's rmse: 2.15952\n",
      "[1900]\tvalid_0's rmse: 2.15505\n",
      "[2000]\tvalid_0's rmse: 2.15058\n",
      "[2100]\tvalid_0's rmse: 2.14359\n",
      "[2200]\tvalid_0's rmse: 2.13794\n",
      "[2300]\tvalid_0's rmse: 2.13262\n",
      "[2400]\tvalid_0's rmse: 2.12769\n",
      "[2500]\tvalid_0's rmse: 2.1229\n",
      "[2600]\tvalid_0's rmse: 2.1172\n",
      "[2700]\tvalid_0's rmse: 2.11114\n",
      "[2800]\tvalid_0's rmse: 2.10638\n",
      "[2900]\tvalid_0's rmse: 2.10206\n",
      "[3000]\tvalid_0's rmse: 2.09767\n",
      "Train CA_2 HOBBIES\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 1.7105\n",
      "[200]\tvalid_0's rmse: 1.64561\n",
      "[300]\tvalid_0's rmse: 1.60863\n",
      "[400]\tvalid_0's rmse: 1.57242\n",
      "[500]\tvalid_0's rmse: 1.53577\n",
      "[600]\tvalid_0's rmse: 1.50163\n",
      "[700]\tvalid_0's rmse: 1.46956\n",
      "[800]\tvalid_0's rmse: 1.43805\n",
      "[900]\tvalid_0's rmse: 1.4065\n",
      "[1000]\tvalid_0's rmse: 1.37693\n",
      "[1100]\tvalid_0's rmse: 1.34912\n",
      "[1200]\tvalid_0's rmse: 1.31803\n",
      "[1300]\tvalid_0's rmse: 1.29236\n",
      "[1400]\tvalid_0's rmse: 1.26743\n",
      "[1500]\tvalid_0's rmse: 1.24402\n",
      "[1600]\tvalid_0's rmse: 1.2211\n",
      "[1700]\tvalid_0's rmse: 1.19869\n",
      "[1800]\tvalid_0's rmse: 1.18022\n",
      "[1900]\tvalid_0's rmse: 1.16087\n",
      "[2000]\tvalid_0's rmse: 1.1418\n",
      "[2100]\tvalid_0's rmse: 1.12439\n",
      "[2200]\tvalid_0's rmse: 1.10701\n",
      "[2300]\tvalid_0's rmse: 1.09325\n",
      "[2400]\tvalid_0's rmse: 1.08055\n",
      "[2500]\tvalid_0's rmse: 1.06703\n",
      "[2600]\tvalid_0's rmse: 1.05348\n",
      "[2700]\tvalid_0's rmse: 1.04145\n",
      "[2800]\tvalid_0's rmse: 1.03075\n",
      "[2900]\tvalid_0's rmse: 1.01829\n",
      "[3000]\tvalid_0's rmse: 1.00826\n",
      "Train CA_2 HOUSEHOLD\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 1.40939\n",
      "[200]\tvalid_0's rmse: 1.34879\n",
      "[300]\tvalid_0's rmse: 1.33299\n",
      "[400]\tvalid_0's rmse: 1.32152\n",
      "[500]\tvalid_0's rmse: 1.31132\n",
      "[600]\tvalid_0's rmse: 1.30184\n",
      "[700]\tvalid_0's rmse: 1.29437\n",
      "[800]\tvalid_0's rmse: 1.28696\n",
      "[900]\tvalid_0's rmse: 1.28016\n",
      "[1000]\tvalid_0's rmse: 1.27403\n",
      "[1100]\tvalid_0's rmse: 1.26746\n",
      "[1200]\tvalid_0's rmse: 1.26078\n",
      "[1300]\tvalid_0's rmse: 1.25508\n",
      "[1400]\tvalid_0's rmse: 1.24882\n",
      "[1500]\tvalid_0's rmse: 1.2431\n",
      "[1600]\tvalid_0's rmse: 1.2378\n",
      "[1700]\tvalid_0's rmse: 1.23276\n",
      "[1800]\tvalid_0's rmse: 1.22776\n",
      "[1900]\tvalid_0's rmse: 1.22258\n",
      "[2000]\tvalid_0's rmse: 1.2182\n",
      "[2100]\tvalid_0's rmse: 1.21374\n",
      "[2200]\tvalid_0's rmse: 1.20922\n",
      "[2300]\tvalid_0's rmse: 1.20489\n",
      "[2400]\tvalid_0's rmse: 1.2009\n",
      "[2500]\tvalid_0's rmse: 1.197\n",
      "[2600]\tvalid_0's rmse: 1.19269\n",
      "[2700]\tvalid_0's rmse: 1.18841\n",
      "[2800]\tvalid_0's rmse: 1.18413\n",
      "[2900]\tvalid_0's rmse: 1.18026\n",
      "[3000]\tvalid_0's rmse: 1.17642\n",
      "Train CA_2 FOODS\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.51865\n",
      "[200]\tvalid_0's rmse: 2.2429\n",
      "[300]\tvalid_0's rmse: 2.18822\n",
      "[400]\tvalid_0's rmse: 2.16313\n",
      "[500]\tvalid_0's rmse: 2.14247\n",
      "[600]\tvalid_0's rmse: 2.12321\n",
      "[700]\tvalid_0's rmse: 2.10913\n",
      "Train CA_3 HOBBIES\n",
      "[LightGBM] [Warning] feature_fraction is set=0.5, colsample_bytree=1.0 will be ignored. Current value: feature_fraction=0.5\n",
      "[LightGBM] [Warning] min_data_in_leaf is set=255, min_child_samples=20 will be ignored. Current value: min_data_in_leaf=255\n",
      "[100]\tvalid_0's rmse: 2.14383\n",
      "[200]\tvalid_0's rmse: 2.07174\n",
      "[300]\tvalid_0's rmse: 2.02436\n",
      "[400]\tvalid_0's rmse: 1.97643\n",
      "[500]\tvalid_0's rmse: 1.92835\n",
      "[900]\tvalid_0's rmse: 1.76322\n",
      "[1000]\tvalid_0's rmse: 1.72243\n",
      "[1100]\tvalid_0's rmse: 1.68652\n",
      "[1200]\tvalid_0's rmse: 1.653\n",
      "[1300]\tvalid_0's rmse: 1.62103\n",
      "[1400]\tvalid_0's rmse: 1.5967\n",
      "[1500]\tvalid_0's rmse: 1.56518\n",
      "[1600]\tvalid_0's rmse: 1.54042\n",
      "[1700]\tvalid_0's rmse: 1.51367\n",
      "[1800]\tvalid_0's rmse: 1.49212\n",
      "[1900]\tvalid_0's rmse: 1.47125\n",
      "[2000]\tvalid_0's rmse: 1.45122\n",
      "[2100]\tvalid_0's rmse: 1.43232\n",
      "[2200]\tvalid_0's rmse: 1.41607\n",
      "[2300]\tvalid_0's rmse: 1.40143\n",
      "[2400]\tvalid_0's rmse: 1.38171\n",
      "[2500]\tvalid_0's rmse: 1.36624\n",
      "[2600]\tvalid_0's rmse: 1.35268\n",
      "[2700]\tvalid_0's rmse: 1.33424\n",
      "[2800]\tvalid_0's rmse: 1.3227\n",
      "[2900]\tvalid_0's rmse: 1.308\n",
      "[3000]\tvalid_0's rmse: 1.29579\n"
     ]
    }
   ],
   "source": [
    "########################### Train Models\n",
    "#################################################################################\n",
    "from lightgbm import LGBMRegressor\n",
    "from gluonts.model.rotbaum._model import QRX\n",
    "\n",
    "for store_id in STORES:\n",
    "    for state_id in CATS:\n",
    "        print(\"Train\", store_id, state_id)\n",
    "\n",
    "        grid_df, features_columns = get_data_by_store(store_id, state_id)\n",
    "\n",
    "        train_mask = grid_df[\"d\"] <= END_TRAIN\n",
    "        valid_mask = train_mask & (grid_df[\"d\"] > (END_TRAIN - P_HORIZON))\n",
    "        preds_mask = (grid_df[\"d\"] > (END_TRAIN - 100)) & (\n",
    "            grid_df[\"d\"] <= END_TRAIN + P_HORIZON\n",
    "        )\n",
    "\n",
    "        #        train_data = lgb.Dataset(grid_df[train_mask][features_columns],\n",
    "        #                           label=grid_df[train_mask][TARGET])\n",
    "\n",
    "        #        valid_data = lgb.Dataset(grid_df[valid_mask][features_columns],\n",
    "        #                           label=grid_df[valid_mask][TARGET])\n",
    "\n",
    "        seed_everything(SEED)\n",
    "        estimator = QRX(\n",
    "            model=LGBMRegressor(**lgb_params),  # lgb_wrapper(**lgb_params),\n",
    "            min_bin_size=200,\n",
    "        )\n",
    "        estimator.fit(\n",
    "            grid_df[train_mask][features_columns],\n",
    "            grid_df[train_mask][TARGET],\n",
    "            max_sample_size=1000000,\n",
    "            seed=SEED,\n",
    "            eval_set=(\n",
    "                grid_df[valid_mask][features_columns],\n",
    "                grid_df[valid_mask][TARGET],\n",
    "            ),\n",
    "            verbose=100,\n",
    "            x_train_is_dataframe=True,\n",
    "        )\n",
    "\n",
    "        #        estimator = lgb.train(lgb_params,\n",
    "        #                              train_data,\n",
    "        #                              valid_sets = [valid_data],\n",
    "        #                              verbose_eval = 100\n",
    "        #\n",
    "        #                              )\n",
    "\n",
    "        #        display(pd.DataFrame({'name':estimator.feature_name(),\n",
    "        #                              'imp':estimator.feature_importance()}).sort_values('imp',ascending=False).head(25))\n",
    "        grid_df = grid_df[preds_mask].reset_index(drop=True)\n",
    "        keep_cols = [col for col in list(grid_df) if \"_tmp_\" not in col]\n",
    "        grid_df = grid_df[keep_cols]\n",
    "\n",
    "        d_sales = grid_df[[\"d\", \"sales\"]]\n",
    "        substitute = d_sales[\"sales\"].values\n",
    "        substitute[(d_sales[\"d\"] > END_TRAIN)] = np.nan\n",
    "        grid_df[\"sales\"] = substitute\n",
    "\n",
    "        grid_df.to_pickle(\n",
    "            processed_data_dir + \"test_\" + store_id + \"_\" + state_id + \".pkl\"\n",
    "        )\n",
    "\n",
    "        model_name = (\n",
    "            model_dir\n",
    "            + \"lgb_model_\"\n",
    "            + store_id\n",
    "            + \"_\"\n",
    "            + state_id\n",
    "            + \"_v\"\n",
    "            + str(VER)\n",
    "            + \".bin\"\n",
    "        )\n",
    "        pickle.dump(estimator, open(model_name, \"wb\"))\n",
    "\n",
    "        del grid_df, d_sales, substitute, estimator  # , train_data, valid_data\n",
    "        gc.collect()\n",
    "\n",
    "        MODEL_FEATURES = features_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "conda_mxnet_p36",
   "language": "python",
   "name": "conda_mxnet_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
